#coding=utf-8

import urllib2
from BeautifulSoup import BeautifulSoup

class FetchContent(object):
    
    def __init__(self, url, fileName):
        self.url = url
        self.fileName = fileName
        self.user_agent = 'User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; \
        Windows NT 5.1; SV1; CNCDialer; .NET CLR 2.0.50727; .NET CLR \
        3.0.04506.648; .NET CLR 3.5.21022)'

    def fetch(self):
        headers={'User-Agent':self.user_agent}
        req = urllib2.Request(self.url)
        try:
            res = urllib2.urlopen(req)
        except urllib2.URLError,e:
            print e.reason
        print "fileName:%s"%self.fileName
        f = open("c:\\" + self.fileName + ".txt","wb")
        soup = BeautifulSoup(res, fromEncoding="UTF-8")
        f.write(soup.head.title.renderContents())
        f.write('\r\n')
        divContent = soup.find('div',{'class':'content'})
        divSoup = BeautifulSoup(str(divContent.renderContents()), fromEncoding="UTF-8")
        divElem = divSoup.findAll('div')
        for d in divElem:
            d.extract()
        pElem = divSoup.findAll('p',{'align':'center'})
        for p in pElem:
            p.extract()
        scriptElem = divSoup.findAll('script')
        for s in scriptElem:
            s.extract()
        for content in divSoup:
            if len(content) == 1:
                text = str(content)
                if text.strip() != '':
                    f.write(text)
        f.close()

if __name__ == '__main__':
    fetch = FetchContent('','')
    fetch.fetch()